package net.bwie.realtime.jtp.dwd.log.job;

import com.alibaba.fastjson.JSON;
import net.bwie.realtime.jtp.KafkaUtil;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SideOutputDataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;
import org.apache.kafka.common.protocol.types.Field;

/**
 * DWD层数据应用开发：将ODS层采集原始日志数据，进行分类处理，存储Kafka队列。
 * 数据流向：kafka -> flink datastream -> kafka
 */
public class JtpAppLogEtlJob {
    public static void main(String[] args) throws Exception {
        // 1.执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //  设置并行度
        env.setParallelism(1);

        // 2.数据源
        DataStream<String> kafka = KafkaUtil.consumerKafka(env, "topic-log");
        // 输出
        // kafka.print();

        // 3.数据转换

        DataStream<String> proceStream = processLog(kafka);
        proceStream.print();

        //4.数据输出


        // 5.触发执行
        env.execute("JtpLogEtlJob");
    }

    /**
     * 对实时获取APP流量日志数据进行ETL处理，并且进行分流
     * 1-数据清洗
     * 2-新老访客状态标记修复
     * 3-数据分流
     *
     * @param stream app日志流
     */
    private static DataStream<String> processLog(DataStream<String> stream) {
        // 1-数据清洗
        DataStream<String> jsonStream = appLogCleaned(stream);

        // 2-新老访客状态标记修复


        //3-数据分流
        DataStream<String> pagStream = splitStream(jsonStream);

        // 4-返回数据流
        return pagStream;
    }


    // 1-数据清洗
    private static DataStream<String> appLogCleaned(DataStream<String> stream) {
        // 1.创建脏数据 侧边流输出 标记
        OutputTag<String> dirtyTag = new OutputTag<String>("dirty-log"){};

        // 2.对数据进行清洗处理
        SingleOutputStreamOperator<String> cleanedStream = stream.process(new ProcessFunction<String, String>() {
            @Override
            public void processElement(String s, ProcessFunction<String, String>.Context context, Collector<String> collector) throws Exception {
                try {
                    // a.解析 json 数据
                    JSON.parseObject(s);
                    // b.输出 解析过的数据
                    collector.collect(s);
                }
                // c.侧边流输出脏数据
                catch (Exception e){
                    context.output(dirtyTag, s);
                }
            }
        });
        //3.将数据转换成 json格式 存储到 kafka中
        SideOutputDataStream<String> sideOutput = cleanedStream.getSideOutput(dirtyTag);
        KafkaUtil.producerKafka(sideOutput, "dwd-traffic-dirty-log");

        // 4.返回清洗过的数据
        return cleanedStream;
    }



    // 3-数据分流
    private static DataStream<String> splitStream(DataStream<String> stream) {
        // 1. 侧边流输出标记
        final OutputTag<String> errorTag = new OutputTag<String>("error-log"){};
        final OutputTag<String> startTag = new OutputTag<String>("start-log"){};
        final OutputTag<String> displayTag = new OutputTag<String>("display-log"){};
        final OutputTag<String> actionTag = new OutputTag<String>("action-log"){};



        // 2. 日志分流处理
        SingleOutputStreamOperator<Object> pageStream = stream.process(
                new AppLogSplitProcessFunction(errorTag, startTag, displayTag, actionTag)
        );


        return null;
    }

}
